In [1]:
%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt

pd.options.display.float_format = '{:.4f}'.format

import datasets

In [2]:
df_nids_human = pd.read_csv("/home/z/Python/CiscoAML/data/nids-aml-human-queen-072620.csv", header=0)
df_nids_auto = pd.read_csv("/home/z/Python/CiscoAML/data/nids-aml-auto-queen-072620.csv", header=0)

df_cicids = pd.read_csv("/home/z/Python/CiscoAML/data/cicids-queen-072020.csv", header=0, low_memory=True)
df_ctu13 = pd.read_csv("/home/z/Python/CiscoAML/data/ctu-13-queen-072620.csv", header=0)
In [3]:
BYTES_TO_MB_DIV = 0.000001
def print_memory_usage_of_data_frame(df):
    mem = round(df.memory_usage().sum() * BYTES_TO_MB_DIV, 3) 
    print("Memory usage is " + str(mem) + " MB")
In [4]:
print_memory_usage_of_data_frame(df_nids_human)

Memory usage is 823.121 MB
In [5]:
print_memory_usage_of_data_frame(df_nids_auto)

Memory usage is 785.629 MB
In [6]:
print_memory_usage_of_data_frame(df_cicids)

Memory usage is 1832.464 MB
In [7]:
print_memory_usage_of_data_frame(df_ctu13)

Memory usage is 199.614 MB

check columns with all 0s for the amount of zeros

Methodology - features in the RF heatmap that are not >=0.01 are effectively 0. Feature must meet this criteria across all datasets.

Features identified as 'All 0s' include: Total Fwd Packet Total Bwd packets Total Length of Bwd Packet Fwd Packet Length Std Flow Bytes/s Flow IAT Min Fwd IAT Total Bwd IAT Total Bwd IAT Max Bwd IAT Min Fwd PSH Flags Bwd PSH Flags Fwd URG Flags Bwd URG Flags Fwd Packets/s Packet Length Min Fin Flag Count PSH Flag Count ACK Flag Count URG Flag Count CWE Flag Count ECE Flag Count Down/Up Ratio Fwd Bytes/Bulk Avg Fwd Packet/Bulk Avg Fwd Bulk Rate Avg Bwd Bytes/Bulk Avg Bwd Packet/Bulk Avg Subflow Fwd Packets Fwd Act Data Pkts Active Mean Active Std Active Max Active Min Idle Std

This is almost half the feature set.

Check for zero and near-zero variance

NIDS-AML-Human

In [18]:
# number of distinct observations over requested axis
df_nids_human_nuniq = df_nids_human.nunique()
In [9]:
counts = df_nids_human.nunique()
to_del = [i for i,v in enumerate(counts) if v == 1]
print(to_del)

#df_nids_human.drop(to_del, axis=1, inplace=True)

[34, 35, 36, 51, 58, 59, 60, 61, 66, 72, 73, 74, 75]
In [19]:
# Calculate variance with var()
df_nids_human_var = df_nids_human.var()
In [23]:
df_nids_human_nzv = pd.concat([df_nids_human_nuniq, df_nids_human_var], axis=1)
In [26]:
df_nids_human_nzv.columns = ['nunique', 'var']
In [27]:
df_nids_human_nzv.head()
Out[27]:
nunique var
Src Port 52206 436473282.0219
Dst Port 54858 543274248.4261
Protocol 3 25.5362
Timestamp 197586 59022916054903.9453
Flow Duration 323777 383384969852032192.0000
In [30]:
feature = ['Src Port', 'Dst Port', 'Protocol', 'Timestamp','Flow Duration', 'Total Fwd Packet',
       'Total Bwd packets', 'Total Length of Fwd Packet',
       'Total Length of Bwd Packet', 'Fwd Packet Length Max',
       'Fwd Packet Length Min', 'Fwd Packet Length Mean',
       'Fwd Packet Length Std', 'Bwd Packet Length Max',
       'Bwd Packet Length Min', 'Bwd Packet Length Mean',
       'Bwd Packet Length Std', 'Flow Bytes/s', 'Flow Packets/s','Flow IAT Mean', 'Flow IAT Std',
       'Flow IAT Max', 'Flow IAT Min', 'Fwd IAT Total', 'Fwd IAT Mean',
       'Fwd IAT Std', 'Fwd IAT Max', 'Fwd IAT Min', 'Bwd IAT Total',
       'Bwd IAT Mean', 'Bwd IAT Std', 'Bwd IAT Max', 'Bwd IAT Min',
       'Fwd PSH Flags', 'Bwd PSH Flags', 'Fwd URG Flags', 'Bwd URG Flags',
       'Fwd Header Length', 'Bwd Header Length', 'Fwd Packets/s',
       'Bwd Packets/s', 'Packet Length Min', 'Packet Length Max',
       'Packet Length Mean', 'Packet Length Std', 'Packet Length Variance',
       'FIN Flag Count', 'SYN Flag Count', 'RST Flag Count', 'PSH Flag Count',
       'ACK Flag Count', 'URG Flag Count', 'CWE Flag Count', 'ECE Flag Count',
       'Down/Up Ratio', 'Average Packet Size', 'Fwd Segment Size Avg',
       'Bwd Segment Size Avg', 'Fwd Bytes/Bulk Avg', 'Fwd Packet/Bulk Avg',
       'Fwd Bulk Rate Avg', 'Bwd Bytes/Bulk Avg', 'Bwd Packet/Bulk Avg',
       'Bwd Bulk Rate Avg', 'Subflow Fwd Packets', 'Subflow Fwd Bytes',
       'Subflow Bwd Packets', 'Subflow Bwd Bytes', 'FWD Init Win Bytes',
       'Bwd Init Win Bytes', 'Fwd Act Data Pkts', 'Fwd Seg Size Min',
       'Active Mean', 'Active Std', 'Active Max', 'Active Min', 'Idle Mean',
       'Idle Std', 'Idle Max', 'Idle Min']
In [34]:
import seaborn as sns
sns.set(font_scale=2)

y_axis_labels = feature

# create seaborn heatmap with required label
p = sns.heatmap(df_nids_human_nzv, yticklabels=y_axis_labels, vmin=0, vmax=5000,
                cmap='gist_ncar', annot=True, fmt=".4f",annot_kws={'size':25}, linewidths=0.75)

NIDS-AML-Auto

In [35]:
# number of distinct observations over requested axis
df_nids_auto_nuniq = df_nids_auto.nunique()
In [36]:
counts = df_nids_auto.nunique()
to_del = [i for i,v in enumerate(counts) if v == 1]
print(to_del)

[34, 35, 36, 51, 58, 59, 60, 61, 66, 72, 73, 74, 75]
In [37]:
# Calculate variance with var()
df_nids_auto_var = df_nids_auto.var()
In [38]:
df_nids_auto_nzv = pd.concat([df_nids_auto_nuniq, df_nids_auto_var], axis=1)
In [39]:
df_nids_auto_nzv.columns = ['nunique', 'var']
In [40]:
sns.set(font_scale=2)

y_axis_labels = feature

# create seaborn heatmap with required label
p = sns.heatmap(df_nids_auto_nzv, yticklabels=y_axis_labels, vmin=0, vmax=5000,
                cmap='gist_ncar', annot=True, fmt=".4f",annot_kws={'size':25}, linewidths=0.75)

CTU-13

In [41]:
# number of distinct observations over requested axis
df_ctu13_nuniq = df_ctu13.nunique()
In [42]:
counts = df_ctu13.nunique()
to_del = [i for i,v in enumerate(counts) if v == 1]
print(to_del)

[34, 35, 36, 51, 58, 59, 60, 61, 66, 72, 73, 74, 75]
In [43]:
# Calculate variance with var()
df_ctu13_var = df_ctu13.var()
In [44]:
df_ctu13_nzv = pd.concat([df_ctu13_nuniq, df_ctu13_var], axis=1)
In [45]:
df_ctu13_nzv.columns = ['nunique', 'var']
In [46]:
sns.set(font_scale=2)

y_axis_labels = feature

# create seaborn heatmap with required label
p = sns.heatmap(df_ctu13_nzv, yticklabels=y_axis_labels, vmin=0, vmax=5000,
                cmap='gist_ncar', annot=True, fmt=".4f",annot_kws={'size':25}, linewidths=0.75)

CICIDS2017

In [47]:
df_cicids_nuniq = df_cicids.nunique()
In [48]:
counts = df_cicids.nunique()
to_del = [i for i,v in enumerate(counts) if v == 1]
print(to_del)

[34, 36, 58, 59, 60, 61, 62, 63]
In [49]:
# Calculate variance with var()
df_cicids_var = df_cicids.var()
In [50]:
df_cicids_nzv = pd.concat([df_cicids_nuniq, df_cicids_var], axis=1)
In [51]:
df_cicids_nzv.columns = ['nunique', 'var']
In [52]:
sns.set(font_scale=2)

y_axis_labels = feature

# create seaborn heatmap with required label
p = sns.heatmap(df_cicids_nzv, yticklabels=y_axis_labels, vmin=0, vmax=5000,
                cmap='gist_ncar', annot=True, fmt=".4f",annot_kws={'size':25}, linewidths=0.75)
In [53]:
df_overall = pd.concat([df_nids_human_nzv, df_nids_auto_nzv, df_ctu13_nzv, df_cicids_nzv], axis=1)
In [54]:
df_overall.columns = ['nunique-human', 'var-human','nunique-auto', 'var-auto','nunique-ctu13', 'var-ctu13','nunique-cicids', 'var-cicids']
In [55]:
sns.set(font_scale=2)

y_axis_labels = feature

# create seaborn heatmap with required label
p = sns.heatmap(df_overall, yticklabels=y_axis_labels, vmin=0, vmax=5000,
                cmap='gist_ncar', annot=True, fmt=".4f",annot_kws={'size':25}, linewidths=0.75)
In [ ]: